{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Buriy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from os.path import (\n",
    "#     join,\n",
    "#     expanduser\n",
    "# )\n",
    "# from itertools import islice as head\n",
    "# import tarfile\n",
    "# from io import BytesIO\n",
    "\n",
    "# from tqdm import tqdm_notebook as log_progress\n",
    "\n",
    "\n",
    "# def source_path(filename, dir='~/proj/corus-data/buriy/'):\n",
    "#     return join(\n",
    "#         expanduser(dir),\n",
    "#         filename\n",
    "#     )\n",
    "\n",
    "\n",
    "# def target_path(filename, dir='buriy'):\n",
    "#     return join(dir, filename)\n",
    "\n",
    "\n",
    "# def top_lines(file, count):\n",
    "#     lines = head(file, count)\n",
    "#     return b''.join(lines)\n",
    "\n",
    "\n",
    "# def sample(source, target, count):\n",
    "#     with tarfile.open(source) as source, tarfile.open(target, 'w:bz2') as target:\n",
    "#         for member in log_progress(head(source, count)):\n",
    "#             if not member.isfile():\n",
    "#                 continue\n",
    "#             file = source.extractfile(member)\n",
    "#             data = top_lines(file, 2)\n",
    "#             member.size = len(data)\n",
    "#             file = BytesIO(data)\n",
    "#             target.addfile(member, file)\n",
    "\n",
    "\n",
    "# filenames = [\n",
    "#     'lenta.tar.bz2',\n",
    "#     'news-articles-2014.tar.bz2',\n",
    "#     'news-articles-2015-part1.tar.bz2',\n",
    "#     'news-articles-2015-part2.tar.bz2',\n",
    "#     'webhose-2016.tar.bz2',\n",
    "# ]\n",
    "# for filename in filenames:\n",
    "#     print(filename)\n",
    "#     sample(source_path(filename), target_path(filename), 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Taiga"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from os.path import (\n",
    "    join,\n",
    "    expanduser\n",
    ")\n",
    "\n",
    "from tqdm import tqdm_notebook as log_progress\n",
    "\n",
    "from corus.io import (\n",
    "    load_tar,\n",
    "    load_zip\n",
    ")\n",
    "\n",
    "\n",
    "def source_path(filename, dir='~/corus-data/taiga/'):\n",
    "    return join(\n",
    "        expanduser(dir),\n",
    "        filename\n",
    "    )\n",
    "\n",
    "\n",
    "def target_path(filename, dir='taiga'):\n",
    "    return join(dir, filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Offsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# def get_offsets(records, patterns):\n",
    "#     offsets = {}\n",
    "#     for record in records:\n",
    "#         for pattern in patterns:\n",
    "#             if pattern not in offsets and pattern in record.name:\n",
    "#                 offsets[pattern] = record.offset\n",
    "#         if len(offsets) == len(patterns):\n",
    "#             break\n",
    "#     return offsets\n",
    "\n",
    "\n",
    "# patterns = ['meta', '/texts', '/tagged']\n",
    "# files = [\n",
    "#     ('Arzamas.tar.gz', load_tar, patterns),\n",
    "#     ('Fontanka.tar.gz', load_tar, patterns),\n",
    "#     ('Interfax.tar.gz', load_tar, patterns),\n",
    "#     ('KP.tar.gz', load_tar, patterns),\n",
    "#     ('Lenta.tar.gz', load_tar, patterns),\n",
    "#     ('Magazines.tar.gz', load_tar, patterns),\n",
    "#     ('NPlus1.tar.gz', load_tar, patterns),\n",
    "#     ('Subtitles.tar.gz', load_tar, patterns),\n",
    "    \n",
    "#     ('social.tar.gz', load_tar, ['/texts', '/tagged']),\n",
    "\n",
    "#     ('proza_ru.zip', load_zip, patterns),\n",
    "#     ('stihi_ru.zip', load_zip, patterns),\n",
    "# ]\n",
    "# for filename, load, patterns in files:\n",
    "#     path = source_path(filename)\n",
    "#     records = load(path)\n",
    "#     print(filename)\n",
    "#     offsets = get_offsets(log_progress(records), patterns)\n",
    "#     for pattern in patterns:\n",
    "#         print('', offsets.get(pattern), pattern, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Arzamas.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t144896\t/texts\n",
    "# \t5112320\t/tagged\n",
    "\n",
    "# Fontanka.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t306359296\t/texts\n",
    "# \t1394093568\t/tagged\n",
    "\n",
    "# Interfax.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t11447296\t/texts\n",
    "# \t140434432\t/tagged\n",
    "\n",
    "# KP.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t13042176\t/texts\n",
    "# \t126222848\t/tagged\n",
    "\n",
    "# Lenta.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t12800000\t/texts\n",
    "# \t140551168\t/tagged\n",
    "\n",
    "# Magazines.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t7292416\t/texts\n",
    "# \t2390665216\t/tagged\n",
    "\n",
    "# NPlus1.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t1919488\t/texts\n",
    "# \t33988608\t/tagged\n",
    "\n",
    "# Subtitles.tar.gz\n",
    "# \t512\tmeta\n",
    "# \t2113024\t/texts\n",
    "# \t974075904\t/tagged\n",
    "\n",
    "# social.tar.gz\n",
    "# \t3985892864\t/texts\n",
    "# \t1024\t/tagged\n",
    "\n",
    "# proza_ru.zip\n",
    "# \t636\tmeta\n",
    "# \t51432715409\t/texts\n",
    "# \t201377139\t/tagged\n",
    "\n",
    "# stihi_ru.zip\n",
    "# \t899\tmeta\n",
    "# \t22304202421\t/texts\n",
    "# \t381570084\t/tagged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def tar_copy_(lines, target, info):\n",
    "    data = b''.join(lines)\n",
    "    file = BytesIO(data)\n",
    "    info.size = len(data)\n",
    "    target.addfile(info, file)\n",
    "\n",
    "\n",
    "def tar_copy_text(source, target, info, count):\n",
    "    file = source.extractfile(info)\n",
    "    lines = islice(file, count)\n",
    "    tar_copy_(lines, target, info)\n",
    "\n",
    "\n",
    "def tar_copy_meta(source, target, info, pattern, encoding='utf8'):\n",
    "    file = source.extractfile(info)\n",
    "    lines = [\n",
    "        _ for index, _\n",
    "        in enumerate(file)\n",
    "        if pattern in _.decode(encoding) or index == 0\n",
    "    ]\n",
    "    if len(lines) == 1:  # just header\n",
    "        return\n",
    "    tar_copy_(lines, target, info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# def sample(source, target, pattern):\n",
    "#     with tarfile.open(source) as source, tarfile.open(target, 'w:gz') as target:\n",
    "#         texts = 0\n",
    "#         for info in log_progress(source):\n",
    "#             if not info.isfile():\n",
    "#                 continue\n",
    "#             name = info.name\n",
    "#             if 'metadata' in name or 'metatable' in name:\n",
    "#                 tar_copy_meta(source, target, info, pattern)\n",
    "#                 print(name)\n",
    "#             elif ('/tagged' in name or '/text' in name) and pattern in name:\n",
    "#                 tar_copy_text(source, target, info, 100)\n",
    "#                 print(name)\n",
    "#                 texts += 1\n",
    "#             if texts >= 2:\n",
    "#                 break\n",
    "#         source.members = []\n",
    "\n",
    "\n",
    "# FILENAMES = [\n",
    "#     ('Arzamas.tar.gz', '101'),\n",
    "#     ('Fontanka.tar.gz', '20070101001'),\n",
    "#     ('Interfax.tar.gz', 'business199005'),\n",
    "#     ('KP.tar.gz', '10@2598286'),\n",
    "#     ('Lenta.tar.gz', '20091231boeviks'),\n",
    "#     ('NPlus1.tar.gz', '20160915'),\n",
    "#     ('Magazines.tar.gz', '103870'),\n",
    "#     ('Subtitles.tar.gz', 'Pilot.HDTV.XII'),\n",
    "# ]\n",
    "\n",
    "# for filename, pattern in FILENAMES:\n",
    "#     sample(source_path(filename), target_path(filename), pattern)\n",
    "#     print(source, '->', target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# filename = 'social.tar.gz'\n",
    "# source = source_path(filename)\n",
    "# target = source_path(filename)\n",
    "\n",
    "# with tarfile.open(source) as source, tarfile.open(target, 'w:gz') as target:\n",
    "#     for info in log_progress(source):\n",
    "#         if not info.isfile():\n",
    "#             continue\n",
    "#         if '/text' in info.name:\n",
    "#             tar_copy_text(source, target, info, 4)\n",
    "#             print(info.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# import zipfile\n",
    "# from corus.zip import open_zip, list_zip, read_zip\n",
    "\n",
    "\n",
    "# def zip_copy_(lines, target, record):\n",
    "#     data = '\\n'.join(lines)\n",
    "#     target.writestr(record.name, data)\n",
    "\n",
    "\n",
    "# def zip_copy_text(source, target, record, count):\n",
    "#     text = read_zip(source, record)\n",
    "#     lines = text.splitlines()[:count]\n",
    "#     zip_copy_(lines, target, record)\n",
    "\n",
    "\n",
    "# def zip_copy_meta(source, target, record, pattern):\n",
    "#     text = read_zip(source, record)\n",
    "#     lines = text.splitlines()\n",
    "#     lines = [\n",
    "#         _ for index, _\n",
    "#         in enumerate(lines)\n",
    "#         if pattern in _ or index == 0\n",
    "#     ]\n",
    "#     if len(lines) == 1:  # just header\n",
    "#         return\n",
    "#     zip_copy_(lines, target, record)\n",
    "\n",
    "\n",
    "# def sample(source, target, pattern, count):\n",
    "#     with open_zip(source) as source, zipfile.ZipFile(target, 'w') as target:\n",
    "#         texts = 0\n",
    "#         for record in log_progress(list_zip(source)):\n",
    "#             if not record.uncompressed:  # not a file\n",
    "#                 continue\n",
    "#             name = record.name\n",
    "#             if 'metatable' in name:\n",
    "#                 zip_copy_meta(source, target, record, pattern)\n",
    "#                 print(name)\n",
    "#             elif ('/tagged' in name or '/text' in name) and pattern in name:\n",
    "#                 zip_copy_text(source, target, record, count)\n",
    "#                 print(name)\n",
    "#                 texts += 1\n",
    "#             if texts >= 2:\n",
    "#                 break\n",
    "\n",
    "\n",
    "# FILENAMES = [\n",
    "#     ('proza_ru.zip', '20151231005', 10),\n",
    "#     ('stihi_ru.zip', '20151231001', 100)\n",
    "# ]\n",
    "\n",
    "# for filename, pattern, count in FILENAMES:\n",
    "#     source = source_path(filename)\n",
    "#     target = target_path(filename)\n",
    "#     sample(source, target, pattern, count)\n",
    "#     print(source, '->', target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
